#from celery import Celery
from CrawlerQueue import CrawlerQueue
from datetime import datetime,timedelta
from pymongo import MongoClient,errors
from bs4 import BeautifulSoup

import requests
import urllib.parse
import urllib.request
import urllib.robotparser
import math
#import db as mysql
import datetime,json,random,socket,csv,time, re,queue,sys
# from models import TbTieba,TbTiebaUser,TbUser,TbUserEducation,TbUserWork
# from sqlalchemy import Column, String, create_engine
# from sqlalchemy.orm import sessionmaker
# from sqlalchemy.ext.declarative import declarative_base
# from sqlalchemy.sql import text
# import crawler

#app=Celery('tasks', backend='redis://localhost:6379/0', broker='redis://localhost:6379/0')

client=MongoClient('192.168.0.124',27017)
db=client.cache

clientwriter=MongoClient('192.168.0.103',27017)
dbwriter=clientwriter.cache

def getproxy():
    client=MongoClient('192.168.0.124',27017)
    db=client.cache
    a=random.randint(1, db.proxy.find({'status':2}).count()-1)
    ipport=db.proxy.find({'status':2}).limit(1).skip(a-1)
    #print(ipport[0])
    return {'http':'http://%s:%s/'%(ipport[0]['_id'],ipport[0]['port'])}
    #return None
def gethtml(url):
    try:
        proxy=None
        html=requests.get(url,proxies=proxy,timeout=10)
        return html.text
    except Exception as identifier:
        return False  


def crawltiebauser():
    cq=CrawlerQueue(db.tiebaurl)
    tbuser=cq.pop()
    
    while tbuser:
        html=gethtml(tbuser['_id'])
        if html:
            try:
                bs=BeautifulSoup(html,'html.parser')
                namelist = bs.find_all(attrs={'class':'drl_item_name'})
                levellist = bs.find_all(attrs={'class':'drl_item_title'})
                jiangyanlist=bs.find_all(attrs={'class':'drl_item_exp'})
                x=-1
                docs=[]
                for l in namelist:
                    
                    x=x+1
                    
                    docs.append({
                        'tieba':tbuser['tieba'],
                        'name':l.text,
                        'level':int(levellist[x].next.attrs['class'][0][5:]),
                        'jingyan':int(jiangyanlist[x].text),
                        'status':0
                    })
                    
                dbwriter.tieba_user.insert_many(docs)
                print(docs)
                cq.delqueue(tbuser['_id'])
                print(str(x)+",complete")
            except Exception as identifier:
                print(identifier)
                cq.remove(tbuser['_id'])
        else:
            cq.remove(tbuser['_id'])
        tbuser=cq.pop()

if __name__=='__main__':
    crawltiebauser()